{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Dual DQN to Play MoutainCar-v0\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import imp\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "from tensorflow import nn\n",
    "from tensorflow import losses\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "imp.reload(logging)\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:22:06 [INFO] env: <MountainCarEnv<MountainCar-v0>>\n",
      "22:22:06 [INFO] action_space: Discrete(3)\n",
      "22:22:06 [INFO] observation_space: Box(-1.2000000476837158, 0.6000000238418579, (2,), float32)\n",
      "22:22:06 [INFO] reward_range: (-inf, inf)\n",
      "22:22:06 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 30}\n",
      "22:22:06 [INFO] _max_episode_steps: 200\n",
      "22:22:06 [INFO] _elapsed_steps: None\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('MountainCar-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['state', 'action', 'reward', 'next_state', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "\n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DualNet(keras.Model):\n",
    "    def __init__(self, input_size, output_size):\n",
    "        super(DualNet, self).__init__()\n",
    "        self.common_net = keras.Sequential()\n",
    "        self.common_net.add(layers.Dense(64, input_shape=(input_size,), activation=nn.relu))\n",
    "        self.advantage_net = keras.Sequential()\n",
    "        self.advantage_net.add(layers.Dense(32, input_shape=(64,), activation=nn.relu))\n",
    "        self.advantage_net.add(layers.Dense(output_size))\n",
    "        self.v_net = keras.Sequential()\n",
    "        self.v_net.add(layers.Dense(32, input_shape=(64,), activation=nn.relu))\n",
    "        self.v_net.add(layers.Dense(1))\n",
    "\n",
    "    def call(self, s):\n",
    "        h = self.common_net(s)\n",
    "        adv = self.advantage_net(h)\n",
    "        adv = adv - tf.math.reduce_mean(adv, axis=1, keepdims=True)\n",
    "        v = self.v_net(h)\n",
    "        q = v + adv\n",
    "        return q\n",
    "\n",
    "\n",
    "class DualDQNAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = DQNReplayer(10000)\n",
    "\n",
    "        self.evaluate_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                output_size=self.action_n)\n",
    "        self.target_net = self.build_net(\n",
    "                input_size=env.observation_space.shape[0],\n",
    "                output_size=self.action_n)\n",
    "\n",
    "    def build_net(self, input_size, output_size):\n",
    "        net = DualNet(input_size=input_size, output_size=output_size)\n",
    "        optimizer = optimizers.Adam(0.001)\n",
    "        net.compile(loss=losses.mse, optimizer=optimizer)\n",
    "        return net\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.target_net.set_weights(self.evaluate_net.get_weights())\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train' and np.random.rand() < 0.001:\n",
    "            # epsilon-greedy policy in train mode\n",
    "            action = np.random.randint(self.action_n)\n",
    "        else:\n",
    "            qs = self.evaluate_net.predict(observation[np.newaxis])\n",
    "            action = np.argmax(qs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                state, _, _, act, next_state, reward, done, _ = \\\n",
    "                        self.trajectory[-8:]\n",
    "                self.replayer.store(state, act, reward, next_state, done)\n",
    "            if self.replayer.count >= self.replayer.capacity * 0.95:\n",
    "                    # skip first few episodes for speed\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        # replay\n",
    "        states, actions, rewards, next_states, dones = self.replayer.sample(1024)\n",
    "\n",
    "        # train\n",
    "        next_eval_qs = self.evaluate_net.predict(next_states)\n",
    "        next_actions = next_eval_qs.argmax(axis=-1)\n",
    "        next_qs = self.target_net.predict(next_states)\n",
    "        next_max_qs = next_qs[np.arange(next_qs.shape[0]), next_actions]\n",
    "        us = rewards + self.gamma * next_max_qs * (1. - dones)\n",
    "        targets = self.evaluate_net.predict(states)\n",
    "        targets[np.arange(us.shape[0]), actions] = us\n",
    "        self.evaluate_net.fit(states, targets, verbose=0)\n",
    "\n",
    "\n",
    "agent = DualDQNAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:22:07 [INFO] ==== train ====\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:22:18 [DEBUG] train episode 0: reward = -200.00, steps = 200\n",
      "22:22:32 [DEBUG] train episode 1: reward = -200.00, steps = 200\n",
      "22:22:44 [DEBUG] train episode 2: reward = -200.00, steps = 200\n",
      "22:22:59 [DEBUG] train episode 3: reward = -200.00, steps = 200\n",
      "22:23:13 [DEBUG] train episode 4: reward = -200.00, steps = 200\n",
      "22:23:27 [DEBUG] train episode 5: reward = -200.00, steps = 200\n",
      "22:23:46 [DEBUG] train episode 6: reward = -200.00, steps = 200\n",
      "22:24:08 [DEBUG] train episode 7: reward = -200.00, steps = 200\n",
      "22:24:25 [DEBUG] train episode 8: reward = -200.00, steps = 200\n",
      "22:24:42 [DEBUG] train episode 9: reward = -200.00, steps = 200\n",
      "22:25:09 [DEBUG] train episode 10: reward = -200.00, steps = 200\n",
      "22:25:26 [DEBUG] train episode 11: reward = -200.00, steps = 200\n",
      "22:25:40 [DEBUG] train episode 12: reward = -200.00, steps = 200\n",
      "22:25:54 [DEBUG] train episode 13: reward = -200.00, steps = 200\n",
      "22:26:07 [DEBUG] train episode 14: reward = -200.00, steps = 200\n",
      "22:26:20 [DEBUG] train episode 15: reward = -200.00, steps = 200\n",
      "22:26:34 [DEBUG] train episode 16: reward = -200.00, steps = 200\n",
      "22:26:49 [DEBUG] train episode 17: reward = -200.00, steps = 200\n",
      "22:27:03 [DEBUG] train episode 18: reward = -200.00, steps = 200\n",
      "22:27:16 [DEBUG] train episode 19: reward = -200.00, steps = 200\n",
      "22:27:31 [DEBUG] train episode 20: reward = -200.00, steps = 200\n",
      "22:27:46 [DEBUG] train episode 21: reward = -200.00, steps = 200\n",
      "22:28:00 [DEBUG] train episode 22: reward = -200.00, steps = 200\n",
      "22:28:15 [DEBUG] train episode 23: reward = -200.00, steps = 200\n",
      "22:28:30 [DEBUG] train episode 24: reward = -200.00, steps = 200\n",
      "22:28:44 [DEBUG] train episode 25: reward = -200.00, steps = 200\n",
      "22:28:59 [DEBUG] train episode 26: reward = -200.00, steps = 200\n",
      "22:29:13 [DEBUG] train episode 27: reward = -200.00, steps = 200\n",
      "22:29:27 [DEBUG] train episode 28: reward = -200.00, steps = 200\n",
      "22:29:40 [DEBUG] train episode 29: reward = -200.00, steps = 200\n",
      "22:29:53 [DEBUG] train episode 30: reward = -200.00, steps = 200\n",
      "22:30:06 [DEBUG] train episode 31: reward = -200.00, steps = 200\n",
      "22:30:19 [DEBUG] train episode 32: reward = -200.00, steps = 200\n",
      "22:30:37 [DEBUG] train episode 33: reward = -200.00, steps = 200\n",
      "22:30:58 [DEBUG] train episode 34: reward = -200.00, steps = 200\n",
      "22:31:12 [DEBUG] train episode 35: reward = -200.00, steps = 200\n",
      "22:31:26 [DEBUG] train episode 36: reward = -200.00, steps = 200\n",
      "22:31:41 [DEBUG] train episode 37: reward = -200.00, steps = 200\n",
      "22:31:55 [DEBUG] train episode 38: reward = -200.00, steps = 200\n",
      "22:32:12 [DEBUG] train episode 39: reward = -200.00, steps = 200\n",
      "22:32:26 [DEBUG] train episode 40: reward = -200.00, steps = 200\n",
      "22:32:40 [DEBUG] train episode 41: reward = -200.00, steps = 200\n",
      "22:32:55 [DEBUG] train episode 42: reward = -200.00, steps = 200\n",
      "22:33:12 [DEBUG] train episode 43: reward = -200.00, steps = 200\n",
      "22:33:26 [DEBUG] train episode 44: reward = -200.00, steps = 200\n",
      "22:33:40 [DEBUG] train episode 45: reward = -200.00, steps = 200\n",
      "22:33:53 [DEBUG] train episode 46: reward = -200.00, steps = 200\n",
      "22:34:29 [DEBUG] train episode 47: reward = -200.00, steps = 200\n",
      "22:36:05 [DEBUG] train episode 48: reward = -200.00, steps = 200\n",
      "22:38:02 [DEBUG] train episode 49: reward = -200.00, steps = 200\n",
      "22:40:03 [DEBUG] train episode 50: reward = -200.00, steps = 200\n",
      "22:42:20 [DEBUG] train episode 51: reward = -200.00, steps = 200\n",
      "22:44:42 [DEBUG] train episode 52: reward = -200.00, steps = 200\n",
      "22:47:03 [DEBUG] train episode 53: reward = -200.00, steps = 200\n",
      "22:49:07 [DEBUG] train episode 54: reward = -200.00, steps = 200\n",
      "22:51:20 [DEBUG] train episode 55: reward = -200.00, steps = 200\n",
      "22:53:29 [DEBUG] train episode 56: reward = -200.00, steps = 200\n",
      "22:55:46 [DEBUG] train episode 57: reward = -200.00, steps = 200\n",
      "22:56:43 [DEBUG] train episode 58: reward = -87.00, steps = 87\n",
      "22:58:53 [DEBUG] train episode 59: reward = -200.00, steps = 200\n",
      "23:01:17 [DEBUG] train episode 60: reward = -200.00, steps = 200\n",
      "23:03:43 [DEBUG] train episode 61: reward = -200.00, steps = 200\n",
      "23:06:02 [DEBUG] train episode 62: reward = -200.00, steps = 200\n",
      "23:08:17 [DEBUG] train episode 63: reward = -200.00, steps = 200\n",
      "23:10:37 [DEBUG] train episode 64: reward = -200.00, steps = 200\n",
      "23:13:29 [DEBUG] train episode 65: reward = -200.00, steps = 200\n",
      "23:15:04 [DEBUG] train episode 66: reward = -104.00, steps = 104\n",
      "23:18:01 [DEBUG] train episode 67: reward = -200.00, steps = 200\n",
      "23:19:31 [DEBUG] train episode 68: reward = -101.00, steps = 101\n",
      "23:22:29 [DEBUG] train episode 69: reward = -200.00, steps = 200\n",
      "23:25:31 [DEBUG] train episode 70: reward = -200.00, steps = 200\n",
      "23:28:31 [DEBUG] train episode 71: reward = -200.00, steps = 200\n",
      "23:31:33 [DEBUG] train episode 72: reward = -200.00, steps = 200\n",
      "23:34:29 [DEBUG] train episode 73: reward = -200.00, steps = 200\n",
      "23:37:26 [DEBUG] train episode 74: reward = -200.00, steps = 200\n",
      "23:40:30 [DEBUG] train episode 75: reward = -200.00, steps = 200\n",
      "23:43:46 [DEBUG] train episode 76: reward = -200.00, steps = 200\n",
      "23:45:09 [DEBUG] train episode 77: reward = -89.00, steps = 89\n",
      "23:48:31 [DEBUG] train episode 78: reward = -200.00, steps = 200\n",
      "23:51:46 [DEBUG] train episode 79: reward = -200.00, steps = 200\n",
      "23:54:56 [DEBUG] train episode 80: reward = -200.00, steps = 200\n",
      "23:57:55 [DEBUG] train episode 81: reward = -200.00, steps = 200\n",
      "00:00:52 [DEBUG] train episode 82: reward = -200.00, steps = 200\n",
      "00:03:49 [DEBUG] train episode 83: reward = -200.00, steps = 200\n",
      "00:06:43 [DEBUG] train episode 84: reward = -200.00, steps = 200\n",
      "00:09:39 [DEBUG] train episode 85: reward = -200.00, steps = 200\n",
      "00:12:35 [DEBUG] train episode 86: reward = -200.00, steps = 200\n",
      "00:15:31 [DEBUG] train episode 87: reward = -200.00, steps = 200\n",
      "00:18:26 [DEBUG] train episode 88: reward = -200.00, steps = 200\n",
      "00:21:22 [DEBUG] train episode 89: reward = -200.00, steps = 200\n",
      "00:24:18 [DEBUG] train episode 90: reward = -200.00, steps = 200\n",
      "00:27:18 [DEBUG] train episode 91: reward = -200.00, steps = 200\n",
      "00:30:13 [DEBUG] train episode 92: reward = -200.00, steps = 200\n",
      "00:33:14 [DEBUG] train episode 93: reward = -200.00, steps = 200\n",
      "00:36:09 [DEBUG] train episode 94: reward = -200.00, steps = 200\n",
      "00:39:06 [DEBUG] train episode 95: reward = -200.00, steps = 200\n",
      "00:42:05 [DEBUG] train episode 96: reward = -200.00, steps = 200\n",
      "00:45:00 [DEBUG] train episode 97: reward = -200.00, steps = 200\n",
      "00:47:54 [DEBUG] train episode 98: reward = -200.00, steps = 200\n",
      "00:50:48 [DEBUG] train episode 99: reward = -200.00, steps = 200\n",
      "00:53:43 [DEBUG] train episode 100: reward = -200.00, steps = 200\n",
      "00:56:39 [DEBUG] train episode 101: reward = -200.00, steps = 200\n",
      "00:59:34 [DEBUG] train episode 102: reward = -200.00, steps = 200\n",
      "01:02:29 [DEBUG] train episode 103: reward = -200.00, steps = 200\n",
      "01:05:23 [DEBUG] train episode 104: reward = -200.00, steps = 200\n",
      "01:08:18 [DEBUG] train episode 105: reward = -200.00, steps = 200\n",
      "01:11:13 [DEBUG] train episode 106: reward = -200.00, steps = 200\n",
      "01:14:08 [DEBUG] train episode 107: reward = -200.00, steps = 200\n",
      "01:17:03 [DEBUG] train episode 108: reward = -200.00, steps = 200\n",
      "01:19:56 [DEBUG] train episode 109: reward = -200.00, steps = 200\n",
      "01:22:51 [DEBUG] train episode 110: reward = -200.00, steps = 200\n",
      "01:25:44 [DEBUG] train episode 111: reward = -200.00, steps = 200\n",
      "01:28:42 [DEBUG] train episode 112: reward = -200.00, steps = 200\n",
      "01:31:37 [DEBUG] train episode 113: reward = -200.00, steps = 200\n",
      "01:34:33 [DEBUG] train episode 114: reward = -200.00, steps = 200\n",
      "01:37:02 [DEBUG] train episode 115: reward = -171.00, steps = 171\n",
      "01:38:50 [DEBUG] train episode 116: reward = -122.00, steps = 122\n",
      "01:41:46 [DEBUG] train episode 117: reward = -200.00, steps = 200\n",
      "01:44:34 [DEBUG] train episode 118: reward = -191.00, steps = 191\n",
      "01:47:05 [DEBUG] train episode 119: reward = -173.00, steps = 173\n",
      "01:49:59 [DEBUG] train episode 120: reward = -200.00, steps = 200\n",
      "01:52:54 [DEBUG] train episode 121: reward = -200.00, steps = 200\n",
      "01:55:48 [DEBUG] train episode 122: reward = -200.00, steps = 200\n",
      "01:58:35 [DEBUG] train episode 123: reward = -190.00, steps = 190\n",
      "02:01:23 [DEBUG] train episode 124: reward = -192.00, steps = 192\n",
      "02:04:16 [DEBUG] train episode 125: reward = -200.00, steps = 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "02:07:10 [DEBUG] train episode 126: reward = -200.00, steps = 200\n",
      "02:10:05 [DEBUG] train episode 127: reward = -200.00, steps = 200\n",
      "02:13:00 [DEBUG] train episode 128: reward = -200.00, steps = 200\n",
      "02:15:54 [DEBUG] train episode 129: reward = -200.00, steps = 200\n",
      "02:18:47 [DEBUG] train episode 130: reward = -200.00, steps = 200\n",
      "02:21:41 [DEBUG] train episode 131: reward = -200.00, steps = 200\n",
      "02:24:34 [DEBUG] train episode 132: reward = -200.00, steps = 200\n",
      "02:27:30 [DEBUG] train episode 133: reward = -200.00, steps = 200\n",
      "02:29:13 [DEBUG] train episode 134: reward = -117.00, steps = 117\n",
      "02:30:45 [DEBUG] train episode 135: reward = -105.00, steps = 105\n",
      "02:33:38 [DEBUG] train episode 136: reward = -200.00, steps = 200\n",
      "02:35:18 [DEBUG] train episode 137: reward = -114.00, steps = 114\n",
      "02:38:13 [DEBUG] train episode 138: reward = -200.00, steps = 200\n",
      "02:39:58 [DEBUG] train episode 139: reward = -118.00, steps = 118\n",
      "02:42:56 [DEBUG] train episode 140: reward = -200.00, steps = 200\n",
      "02:44:47 [DEBUG] train episode 141: reward = -127.00, steps = 127\n",
      "02:47:04 [DEBUG] train episode 142: reward = -157.00, steps = 157\n",
      "02:48:44 [DEBUG] train episode 143: reward = -114.00, steps = 114\n",
      "02:51:27 [DEBUG] train episode 144: reward = -185.00, steps = 185\n",
      "02:53:04 [DEBUG] train episode 145: reward = -110.00, steps = 110\n",
      "02:54:37 [DEBUG] train episode 146: reward = -106.00, steps = 106\n",
      "02:56:13 [DEBUG] train episode 147: reward = -108.00, steps = 108\n",
      "02:57:28 [DEBUG] train episode 148: reward = -96.00, steps = 96\n",
      "02:58:37 [DEBUG] train episode 149: reward = -87.00, steps = 87\n",
      "03:00:33 [DEBUG] train episode 150: reward = -150.00, steps = 150\n",
      "03:02:10 [DEBUG] train episode 151: reward = -122.00, steps = 122\n",
      "03:04:41 [DEBUG] train episode 152: reward = -194.00, steps = 194\n",
      "03:06:02 [DEBUG] train episode 153: reward = -104.00, steps = 104\n",
      "03:07:34 [DEBUG] train episode 154: reward = -116.00, steps = 116\n",
      "03:10:09 [DEBUG] train episode 155: reward = -200.00, steps = 200\n",
      "03:11:22 [DEBUG] train episode 156: reward = -93.00, steps = 93\n",
      "03:13:18 [DEBUG] train episode 157: reward = -149.00, steps = 149\n",
      "03:15:12 [DEBUG] train episode 158: reward = -147.00, steps = 147\n",
      "03:17:47 [DEBUG] train episode 159: reward = -200.00, steps = 200\n",
      "03:20:21 [DEBUG] train episode 160: reward = -200.00, steps = 200\n",
      "03:21:27 [DEBUG] train episode 161: reward = -86.00, steps = 86\n",
      "03:23:28 [DEBUG] train episode 162: reward = -161.00, steps = 161\n",
      "03:24:35 [DEBUG] train episode 163: reward = -87.00, steps = 87\n",
      "03:27:08 [DEBUG] train episode 164: reward = -200.00, steps = 200\n",
      "03:29:14 [DEBUG] train episode 165: reward = -166.00, steps = 166\n",
      "03:31:46 [DEBUG] train episode 166: reward = -200.00, steps = 200\n",
      "03:32:53 [DEBUG] train episode 167: reward = -88.00, steps = 88\n",
      "03:34:09 [DEBUG] train episode 168: reward = -100.00, steps = 100\n",
      "03:36:39 [DEBUG] train episode 169: reward = -200.00, steps = 200\n",
      "03:39:11 [DEBUG] train episode 170: reward = -200.00, steps = 200\n",
      "03:41:41 [DEBUG] train episode 171: reward = -200.00, steps = 200\n",
      "03:42:44 [DEBUG] train episode 172: reward = -83.00, steps = 83\n",
      "03:44:30 [DEBUG] train episode 173: reward = -140.00, steps = 140\n",
      "03:45:38 [DEBUG] train episode 174: reward = -90.00, steps = 90\n",
      "03:47:33 [DEBUG] train episode 175: reward = -151.00, steps = 151\n",
      "03:49:37 [DEBUG] train episode 176: reward = -163.00, steps = 163\n",
      "03:50:42 [DEBUG] train episode 177: reward = -85.00, steps = 85\n",
      "03:52:21 [DEBUG] train episode 178: reward = -131.00, steps = 131\n",
      "03:53:45 [DEBUG] train episode 179: reward = -111.00, steps = 111\n",
      "03:55:13 [DEBUG] train episode 180: reward = -115.00, steps = 115\n",
      "03:56:49 [DEBUG] train episode 181: reward = -127.00, steps = 127\n",
      "03:58:33 [DEBUG] train episode 182: reward = -136.00, steps = 136\n",
      "04:00:03 [DEBUG] train episode 183: reward = -120.00, steps = 120\n",
      "04:01:36 [DEBUG] train episode 184: reward = -123.00, steps = 123\n",
      "04:03:08 [DEBUG] train episode 185: reward = -121.00, steps = 121\n",
      "04:04:33 [DEBUG] train episode 186: reward = -113.00, steps = 113\n",
      "04:06:02 [DEBUG] train episode 187: reward = -119.00, steps = 119\n",
      "04:07:31 [DEBUG] train episode 188: reward = -118.00, steps = 118\n",
      "04:08:58 [DEBUG] train episode 189: reward = -115.00, steps = 115\n",
      "04:10:42 [DEBUG] train episode 190: reward = -124.00, steps = 124\n",
      "04:12:09 [DEBUG] train episode 191: reward = -115.00, steps = 115\n",
      "04:13:37 [DEBUG] train episode 192: reward = -116.00, steps = 116\n",
      "04:15:06 [DEBUG] train episode 193: reward = -118.00, steps = 118\n",
      "04:16:33 [DEBUG] train episode 194: reward = -115.00, steps = 115\n",
      "04:17:55 [DEBUG] train episode 195: reward = -109.00, steps = 109\n",
      "04:19:17 [DEBUG] train episode 196: reward = -111.00, steps = 111\n",
      "04:20:35 [DEBUG] train episode 197: reward = -116.00, steps = 116\n",
      "04:21:32 [DEBUG] train episode 198: reward = -84.00, steps = 84\n",
      "04:23:29 [DEBUG] train episode 199: reward = -174.00, steps = 174\n",
      "04:24:48 [DEBUG] train episode 200: reward = -119.00, steps = 119\n",
      "04:26:01 [DEBUG] train episode 201: reward = -108.00, steps = 108\n",
      "04:27:04 [DEBUG] train episode 202: reward = -92.00, steps = 92\n",
      "04:28:14 [DEBUG] train episode 203: reward = -105.00, steps = 105\n",
      "04:29:27 [DEBUG] train episode 204: reward = -109.00, steps = 109\n",
      "04:30:36 [DEBUG] train episode 205: reward = -102.00, steps = 102\n",
      "04:31:33 [DEBUG] train episode 206: reward = -85.00, steps = 85\n",
      "04:31:33 [INFO] ==== test ====\n",
      "04:31:44 [DEBUG] test episode 0: reward = -106.00, steps = 106\n",
      "04:31:56 [DEBUG] test episode 1: reward = -106.00, steps = 106\n",
      "04:32:07 [DEBUG] test episode 2: reward = -106.00, steps = 106\n",
      "04:32:18 [DEBUG] test episode 3: reward = -106.00, steps = 106\n",
      "04:32:28 [DEBUG] test episode 4: reward = -88.00, steps = 88\n",
      "04:32:40 [DEBUG] test episode 5: reward = -105.00, steps = 105\n",
      "04:32:53 [DEBUG] test episode 6: reward = -128.00, steps = 128\n",
      "04:33:03 [DEBUG] test episode 7: reward = -90.00, steps = 90\n",
      "04:33:14 [DEBUG] test episode 8: reward = -106.00, steps = 106\n",
      "04:33:26 [DEBUG] test episode 9: reward = -104.00, steps = 104\n",
      "04:33:40 [DEBUG] test episode 10: reward = -138.00, steps = 138\n",
      "04:33:50 [DEBUG] test episode 11: reward = -87.00, steps = 87\n",
      "04:34:02 [DEBUG] test episode 12: reward = -106.00, steps = 106\n",
      "04:34:13 [DEBUG] test episode 13: reward = -106.00, steps = 106\n",
      "04:34:24 [DEBUG] test episode 14: reward = -106.00, steps = 106\n",
      "04:34:36 [DEBUG] test episode 15: reward = -105.00, steps = 105\n",
      "04:34:47 [DEBUG] test episode 16: reward = -104.00, steps = 104\n",
      "04:34:56 [DEBUG] test episode 17: reward = -91.00, steps = 91\n",
      "04:35:08 [DEBUG] test episode 18: reward = -103.00, steps = 103\n",
      "04:35:19 [DEBUG] test episode 19: reward = -107.00, steps = 107\n",
      "04:35:30 [DEBUG] test episode 20: reward = -106.00, steps = 106\n",
      "04:35:42 [DEBUG] test episode 21: reward = -106.00, steps = 106\n",
      "04:35:53 [DEBUG] test episode 22: reward = -106.00, steps = 106\n",
      "04:36:04 [DEBUG] test episode 23: reward = -106.00, steps = 106\n",
      "04:36:16 [DEBUG] test episode 24: reward = -106.00, steps = 106\n",
      "04:36:26 [DEBUG] test episode 25: reward = -91.00, steps = 91\n",
      "04:36:38 [DEBUG] test episode 26: reward = -106.00, steps = 106\n",
      "04:36:49 [DEBUG] test episode 27: reward = -106.00, steps = 106\n",
      "04:36:59 [DEBUG] test episode 28: reward = -105.00, steps = 105\n",
      "04:37:13 [DEBUG] test episode 29: reward = -135.00, steps = 135\n",
      "04:37:24 [DEBUG] test episode 30: reward = -107.00, steps = 107\n",
      "04:37:35 [DEBUG] test episode 31: reward = -105.00, steps = 105\n",
      "04:37:46 [DEBUG] test episode 32: reward = -106.00, steps = 106\n",
      "04:37:59 [DEBUG] test episode 33: reward = -128.00, steps = 128\n",
      "04:38:10 [DEBUG] test episode 34: reward = -105.00, steps = 105\n",
      "04:38:21 [DEBUG] test episode 35: reward = -107.00, steps = 107\n",
      "04:38:32 [DEBUG] test episode 36: reward = -107.00, steps = 107\n",
      "04:38:43 [DEBUG] test episode 37: reward = -105.00, steps = 105\n",
      "04:38:53 [DEBUG] test episode 38: reward = -104.00, steps = 104\n",
      "04:39:02 [DEBUG] test episode 39: reward = -88.00, steps = 88\n",
      "04:39:13 [DEBUG] test episode 40: reward = -106.00, steps = 106\n",
      "04:39:24 [DEBUG] test episode 41: reward = -104.00, steps = 104\n",
      "04:39:35 [DEBUG] test episode 42: reward = -106.00, steps = 106\n",
      "04:39:46 [DEBUG] test episode 43: reward = -105.00, steps = 105\n",
      "04:39:57 [DEBUG] test episode 44: reward = -105.00, steps = 105\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "04:40:08 [DEBUG] test episode 45: reward = -105.00, steps = 105\n",
      "04:40:18 [DEBUG] test episode 46: reward = -104.00, steps = 104\n",
      "04:40:29 [DEBUG] test episode 47: reward = -105.00, steps = 105\n",
      "04:40:40 [DEBUG] test episode 48: reward = -106.00, steps = 106\n",
      "04:40:51 [DEBUG] test episode 49: reward = -104.00, steps = 104\n",
      "04:41:01 [DEBUG] test episode 50: reward = -105.00, steps = 105\n",
      "04:41:12 [DEBUG] test episode 51: reward = -106.00, steps = 106\n",
      "04:41:23 [DEBUG] test episode 52: reward = -106.00, steps = 106\n",
      "04:41:34 [DEBUG] test episode 53: reward = -106.00, steps = 106\n",
      "04:41:45 [DEBUG] test episode 54: reward = -106.00, steps = 106\n",
      "04:41:58 [DEBUG] test episode 55: reward = -128.00, steps = 128\n",
      "04:42:09 [DEBUG] test episode 56: reward = -105.00, steps = 105\n",
      "04:42:20 [DEBUG] test episode 57: reward = -105.00, steps = 105\n",
      "04:42:31 [DEBUG] test episode 58: reward = -104.00, steps = 104\n",
      "04:42:42 [DEBUG] test episode 59: reward = -104.00, steps = 104\n",
      "04:42:55 [DEBUG] test episode 60: reward = -128.00, steps = 128\n",
      "04:43:04 [DEBUG] test episode 61: reward = -88.00, steps = 88\n",
      "04:43:15 [DEBUG] test episode 62: reward = -106.00, steps = 106\n",
      "04:43:25 [DEBUG] test episode 63: reward = -87.00, steps = 87\n",
      "04:43:36 [DEBUG] test episode 64: reward = -107.00, steps = 107\n",
      "04:43:46 [DEBUG] test episode 65: reward = -105.00, steps = 105\n",
      "04:43:57 [DEBUG] test episode 66: reward = -106.00, steps = 106\n",
      "04:44:06 [DEBUG] test episode 67: reward = -88.00, steps = 88\n",
      "04:44:17 [DEBUG] test episode 68: reward = -104.00, steps = 104\n",
      "04:44:28 [DEBUG] test episode 69: reward = -105.00, steps = 105\n",
      "04:44:42 [DEBUG] test episode 70: reward = -133.00, steps = 133\n",
      "04:44:51 [DEBUG] test episode 71: reward = -92.00, steps = 92\n",
      "04:45:02 [DEBUG] test episode 72: reward = -107.00, steps = 107\n",
      "04:45:13 [DEBUG] test episode 73: reward = -107.00, steps = 107\n",
      "04:45:24 [DEBUG] test episode 74: reward = -105.00, steps = 105\n",
      "04:45:35 [DEBUG] test episode 75: reward = -105.00, steps = 105\n",
      "04:45:46 [DEBUG] test episode 76: reward = -107.00, steps = 107\n",
      "04:45:55 [DEBUG] test episode 77: reward = -87.00, steps = 87\n",
      "04:46:06 [DEBUG] test episode 78: reward = -105.00, steps = 105\n",
      "04:46:17 [DEBUG] test episode 79: reward = -106.00, steps = 106\n",
      "04:46:28 [DEBUG] test episode 80: reward = -106.00, steps = 106\n",
      "04:46:39 [DEBUG] test episode 81: reward = -105.00, steps = 105\n",
      "04:46:50 [DEBUG] test episode 82: reward = -106.00, steps = 106\n",
      "04:47:01 [DEBUG] test episode 83: reward = -104.00, steps = 104\n",
      "04:47:10 [DEBUG] test episode 84: reward = -90.00, steps = 90\n",
      "04:47:21 [DEBUG] test episode 85: reward = -107.00, steps = 107\n",
      "04:47:32 [DEBUG] test episode 86: reward = -107.00, steps = 107\n",
      "04:47:43 [DEBUG] test episode 87: reward = -103.00, steps = 103\n",
      "04:47:54 [DEBUG] test episode 88: reward = -106.00, steps = 106\n",
      "04:48:05 [DEBUG] test episode 89: reward = -106.00, steps = 106\n",
      "04:48:18 [DEBUG] test episode 90: reward = -127.00, steps = 127\n",
      "04:48:27 [DEBUG] test episode 91: reward = -88.00, steps = 88\n",
      "04:48:40 [DEBUG] test episode 92: reward = -127.00, steps = 127\n",
      "04:48:50 [DEBUG] test episode 93: reward = -88.00, steps = 88\n",
      "04:49:00 [DEBUG] test episode 94: reward = -106.00, steps = 106\n",
      "04:49:11 [DEBUG] test episode 95: reward = -105.00, steps = 105\n",
      "04:49:22 [DEBUG] test episode 96: reward = -104.00, steps = 104\n",
      "04:49:33 [DEBUG] test episode 97: reward = -107.00, steps = 107\n",
      "04:49:42 [DEBUG] test episode 98: reward = -88.00, steps = 88\n",
      "04:49:53 [DEBUG] test episode 99: reward = -105.00, steps = 105\n",
      "04:49:53 [INFO] average episode reward = -105.21 ± 9.98\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABSOUlEQVR4nO29e7gkV3Uf+ltV3efMOfPWzOiBRiMJEE+ZABqDsLHBtjCKP8cICDcQJ/heJ1cOgXwhyf0cY+69sXONk9i+2JcY67sKfkAIJvihwDUPGYENthGPwch6IhghWRpmpHnrvPt0V637R9WuWnvXru6q7tPPs37fd77uU1VdtU+f7rVX/dZv/TYxMxQKhUKxvRCMewAKhUKhGD00+CsUCsU2hAZ/hUKh2IbQ4K9QKBTbEBr8FQqFYhuiMe4BVMXBgwf5mmuuGfcwFAqFYqrw9a9//SwzH3K3T03wv+aaa3Ds2LFxD0OhUCimCkT0t77tQ6N9iOjFRPRlIrqHiI4R0cvEvncR0XEiepiIXjusMSgUCoXCj2Fm/r8C4BeZ+dNE9GPp768mohcAeDOAFwJ4BoC7iOg5zBwNcSwKhUKhEBhmwZcB7Emf7wVwMn3+OgAfZeYWMz8K4DiAl3ler1AoFIohYZiZ/zsB3ElEv4Zkkvm+dPuVAL4sjjuRbiuAiG4FcCsAHDlyZGgDVSgUiu2GgYI/Ed0F4HLPrncD+BEA/4qZ/4iI/icAvw3gJgDkOd5rMMTMtwO4HQCOHj2qJkQKhUKxRRgo+DPzTWX7iOhDAP5l+usfAPhA+vwEgKvEoYeRU0IKhUKhGAGGyfmfBPCq9PkPA/h2+vwTAN5MRPNEdC2A6wB8dYjjUCgUCoWDYXL+/yuA/4eIGgA2kHL3zPwAEX0MwIMAOgDerkofhUKxlXjw5BLW2xFuuHr/uIcysRha8GfmvwRwQ8m+9wB4z7CurVAotjfe+9lv4czyBj7+jleOeygTC/X2USgUM4d2FKPVicc9jAJOL23gn37wGJY32uMeigZ/hUIxe4iZEXdZpXBzTBPDPU9cxF0PPYXjp1cqHX9xbRPfvbiOON56saMGf4VCMXNgBjolAfOeJy7i+n93J55a2hjxqIDNKJl0uk1MEh/56uP4/v/4+aHcxWjwVygUE4djj53H6eX+g3PMjKgk+J+8uI7NKMbppVbf5+8X5o6jaiK/2uogIGBHc+tDtQZ/hUIxcfjp3/saPvilx/p+fcyMTuSPsGZSaMejp35M8C+bmFystiLsnG+AyNcbOxg0+Ctqox3FeNcf34vvXlwf91AUE4Y/ufck/uvdjw18no1OjPXNGMyM//SZb+JvnrhY6/UxlwdYQ7mUTQ7DRF3aZ6XVwa754YgyNfgrauPEhXX8/lefwJcfOTfuoSgmDP/ty4/jw19+fODzcFqwjRm47c8fwee+ebrmCco5fzMpdKLxZf5VbzpWNfgrJgmcZi1qtqRwcXp5A1HFrLYbTOaeZe/OOdc3I7z8l+/Cnz/snxQSzt8fYXPaZ/Sf4Fanfua/U4O/YlJgvjNVP8Bbgb954iLWN7URfNJxerlVmc/uhpgZkZBruqdc3mjjqaUWHjy1VPr6ssw/p33GyPkr7aOYRmSZ/4iC/0qrgzfe9iX8j3u+O5LrKfrDRjvC8kZn4ODPzGAGYpH5u4mGucTFNX+zVDfO38T8sslhmDCcf9Xvzmqrg53z4VDGosFfURt55j+a6212YnRixmqrM5oLKvrCmeVEOjl48Ed2nqgk8zeTwcW1zZJz5Jn/Z+5/Ep+5/1S2LxpnwTdT+1Q73qh9hoGpWcBdMTnIb8VH8+WJszuNkVxO0SeMLn/Q4G/+3xFz1tnqZsrmmAsVMv/f+ctHAQA3X3+Fda7OGKWedTj/3Rr8FZOCMh52+NfT6D/JME1Tg9Ip2Z1lF9rH/FqW+Zsmr+QOILZ08lnBd4yZfxW7BmZOaR/l/BUTAvPFGxXnby4zBopWUQOnU9pn0Ek6z/xRgfYpz/yBJNB3Yrv4O1apZ+Tv8O1EMb711LK1rZXSnRr8FRODLBMfUTTWzH86YGifQYMqi8zfMDNlBd8y2iendpJOX/lZNecah9SzTO3z6fufxM2/8UWcFn5Dpsalah/FxGDUBd94xHcaiv5gaJ8qn4un19t4/58d9yYQjHyyN0HS/dfLgq/vcyEThk4cO5l/8jiOzN/o/N0xP7W0gZjzuycg4fsBaOavmByMvOCb8b4juZyiT5jAVaWQ+oVvncGv3vkwvnO2aG0sKZu4lPPPM/tVT/+HOYehfGTD1yTYO7hFcRPon15vF7btUqmnYlLAJdnY8K6XPCrtM9moI/U0Qd1XHJbJRS+dPwBcWC0WfbO6QZTQPpGP8x8L7ROl47O3r2wUg/9qKzl213xzKGPR4K+ojVF3+I5aXaToD6frBP8u2TenSXoVnT9QUvQVmb9lE4FJ8fbpnfmvZrSPZv6KCcGoaZhc56/Rf1LRiWKcW22BKPlc9PpfmQDsmyik2qdU5y/i9sX1Lpl/zGhHNudvzjWWgm+Jq+dyGujlRLaiBV/FpGH0mf9or6eoj3Orm2AGLt09D6B39s8iM3ch1WRZ5h/7jwH8ip+c84+t2gEgO3wnx9vHT/towVcxYRi1tw8r7TPxOLuSUD6X7dkBoDefLjPz4j5k+3o1eQH+Rq9umf9YvX1KVvLqVvDV4K+YGIxL6qmZ/+TC0BUHdyWZf6//lczMXXBG+0idv/v67py/vLNwOf9M5z9JnH+a+S951T4a/BUTAvX2Ubi4kGbfB3bOAdiazF/SPmXePvLaZedvi/PIa07SSl4m0Mv6xWqrg4VmiDDY+iUcAQ3+ij4wNm8f5X0mFib7PmAy/56cf2+pZ1RR6unL/KWaKIoZUeQJ/mMwdmuV0D7LG8nfYNM+w3P0BAYM/kT0JiJ6gIhiIjrq7HsXER0nooeJ6LVi+w1EdF+67300jJWJFUOFevsoXBje/eCuqpl/8hh5sm852ZclGmzRPr7MH+k44szfxz3/OC2d5eTIzKVSz2E1eAGDZ/73A3gDgC/KjUT0AgBvBvBCADcD+C0iMn/FbQBuBXBd+nPzgGNQjBjjon2U859cXFhrY3EuxMJc8jXvpfaJu2T+mZ9/hcw/DMir9jGTQ8ujrul27WGCmb20z3o7yv6ep9fs4D+xmT8zP8TMD3t2vQ7AR5m5xcyPAjgO4GVEdAWAPcx8Nyf/nQ8BuGWQMShGD/X2Ubi4uNbGvoUmGik/3Tv4o/S4nK+XOn//MfsX57pm/q120U7B1HlHXfDtxGxNbAam2Htw1xyWW/lKaMNcwhEYHud/JYAnxO8n0m1Xps/d7V4Q0a1EdIyIjp05c2YoA1XUx/gy/5FcTtEHLq5tYt/iHAKqFvy7LajiK/gWM//k913zITbavnMk+zfaUTaeTDI8poKvoXwAezIzDV5X7lsAc87/r26OOfgT0V1EdL/n53XdXubZxl22e8HMtzPzUWY+eujQoV5DVYwIg3j7fOU75/DC//Mz1u1t1esp7TO5uLC2if07m2iEVTP/bmqf3gVf82sYkPdzYTa1OtLQDdl5gdEXfGXwl3+3yfyv3L8AIOf9VzaGS/v0PDMz39THeU8AuEr8fhjAyXT7Yc92xRRBZmZ18fj5NaxuRji/tom9i9UMq0ZNMynq4+J6G1fsXcgy/158erdGK5mhl931me2NIPB+Lsz+Vid3/OzEMcIgzD63o17JazOSE5EI/iLzB0Twn2S1Txd8AsCbiWieiK5FUtj9KjOfArBMRDemKp+3Avj4kMagGBIGoWH6cegs83dRTA4urrWxb7GJRpCElN5NXt0y/+QxyfyT50Wdf/JYJ/PP/IQmIPOXidPyRjH4xzFjab2NPQsTGvyJ6PVEdALAKwB8kojuBABmfgDAxwA8COAzAN7OzGYKfhuADyApAj8C4NODjEExegzScduPSZt2+E424phxcW0T+xfnEKYRpRefXknnb9k7+I9phP7g73L+8lrjavLyUVCAyPz3LwJIgv/ZlRY2oxiH0wlhGBhoWmHmOwDcUbLvPQDe49l+DMD1g1xXMV4M4u3TD4Wj3j6TjeVWBzED+xabCCtn/slj5FHcZJYOFu3jcv6G9iEv/ZjTPsVse1xST4vzt9Q+Cc1jMv+La208cWEdAHA4nRCGAe3wVdRGXdrnoVNLePkv34VzK63sQ1/F8z2/nn1dxWTBSC33yczf+f++64/vxS/+fw9kv3cLwP6Cr3NMGkcbQeAVHpjjfZm/ee2oXT17cv6i4HviwhoA4PD+Cc38FdsTZYtql+HRs6t4aqmFJ5c2+lLuqLfPcNCJYjx4agkvOrxvoPOYJqv9IvOPHD79wVPLltSvm84/08LH5TSh2R4GVLBHlsd34/xHXvAt4/xbHcyFAfYuNDHXCLC03s708FcOMfhr5q+ojbqZf96uX960U+16Gv23Ep+6/0m87v1/hdPLGwOdJ8/8ZZOXfQwzW81YVTj/Kss4lnP+yaPM/LNzjcnbZ7OM89/oYNeOJA8/uHMOTy5t4MSFdRzYOYfFuTFKPRUKF3W9fSRt0w+Fo94+w8GTT6+DGVj3LIBeBZ0oxns/+61MjrhvcQ6nl/yLuEcxWzYMlXT+suDrmUyAlPPvKvXsovYZudSzOBEBdifv9Vfuxb0nnsbh/QtDpXwADf6KPlA3E5eFtm5f+q26nqIaTDCu87+Q+JsTF/Fbf/4IdjQTAmH/4hzOp4upF1feApY22ohiTuWZyXZ/5p8+din45lLPoJCEMOc2Ct3UPu0xZv5yzCsbefB/yZH9+NMHn8LSehs3PvPAUMejtI+iNuoqduQXuJ8eAfX2GQ6MFXK/d1TfePwiAGT2Cnt2NESTlx1Y49TXxixWkicBvgAsC77pFrfgKzJ/d/KSx/oy/3G5erZK1D7LrZz2ecmRfQCSZTGHnflr8FfURu3MX0wW/QRyWTNQbB0MB9/vpPqNxy9iTxq09uxooBEGGedf5sVzIbtmsr1b5h9V8PYJwyLtI49teTj/3M9/jAVfcenVVge708z/RYf3wqzdosFfMXGo6+2T0T5d2vWrXE9pn62FCcQ+tUwVfOPxC/jB5xzCa15wWaZHN6tOuVm1ucbFdZtq8vr5S5qwh7dPM72enMDkkTLbNncjEyH1FF+ApY12lvkvzjXwvMv3ABiuxh9Qzl/RB+oWbaVuO3ayr3rXqz5GRW9ktE8fMfCppQ2cfHoD/+TIfrzlZVdl1E9YkvmbX83dRnedf/JYpcM3byoDUk8569qS83clyqOgfb70yFmsb0b4kedflmX+843AGuPF1Tb2L85lv7/kyD48eGpJM3/F5KG+1DN55AFpH+X8txY551//fTV8/0uO7MPiXAOXpGv3Zpm/8+EwQfzCanJNZnu7hOzoNvvLvH186weUcf4m8890/iPgEX/1zofxzo/eg5VWJwv+C3NhVstodSIstzrZ+wcAP/6iZ+Do1ftx5IBm/ooJQ93MX/K2gxR8lfbZWlxwsvA6uOeJi5gLA7zwGXus7WHJYi6xQ/tUyfyBPECXZv5h8U7D5vw9Bd8Revs8enYVy60O/vDYE1nw39EIs8nMTMAy+L/iWQfwh2/7vqGPTTN/RW3U9fYxx0VxrvOvwzOrt8/WY30zKl1MvAoupv798w17jdmwZDEXE3Bd2sen9pHBu90p4/yT33POX74+fy4tnYuungkNeXalVfp3DoKLa5tZcP/dLz2GjXQs880gG4ORxsrgPypo8FfURl31jezqLSvgVbqeZv5bhovrebdtP+9rFHMW6CXKaB/za363Ae9x7njaUVnmb65XNJKzOX9Pk5f43N710FN4xX/4HJ44v1YYx6B49OwqAOB1L34G/vbcGv7q+DnMNQKElCuUTPCXnP+ooMFfURv1aZ/8+L4snUu03or+Ybh3AF5XzF6ImBEExeBvVvJyzxk5NIe8G3Qh/88m+Jd5+zQ8tA+L4G4v5mLTPkCyuFA7YnztsfOFcQwKE/z/4cuOAAAePLmE+TAAUT4GE/wP7NLgr5gC1OXtfbRPnVqbZv5bD+mz0w/tE6edui7CkpW8XI47k1v2yPw3s8zffyeRqYtiua8k8xeqM4OldCEVU8DeSjx2dhUBJV27B3bOYTOKk8xfLEBj7oQ081dMBcx3p7q3Tz5ZsOcLWPV6Gvy3DqbwCvRn7xAxutI+7v8qU/u4nL9P5+/J/N0hSm8f93pW8Jecf1TM/E3H8TeeuFAYh3u9f/3f78FXvnOu63ESj55bw+H9i5hrBHjOZbsBAHONAAHlXcnnVkzwr7ak6VZCg7+iNury9nIpvjLpXtfraYfvluOC47DJzFYHqg9xzHkwjv20T1mTl4m3rqVET86/pOBrPoO+ZSPlKeXLOp7P7VK6kMpDp5a7GtydW93EH3/ju/j0/U+WHuPi0bMruObgTgDAcy+3g7+sgexdaKIRjj4Ua/BX1Ea/3j4Rc+3X2tfTzH+rcNFy2AQ+++BTuOGXPou1zU7pa37nrx7Fa3/jiwB6F3zLgrVrKeFT+8jEIOf87WMynX/G+ftfb7/GQ/usd7K/577vPu19HQA8+XRie214/F5gZjx2dg3Xplr955ngHwYIgnws51c3x6L0ATT4K/pAXQ5eSjX74e+V8996SM4/YsbJi+tY3uhkq0r58MT5NZy8uJ69pmvmX6LzX92MsNmJLbmlC7mpnPNn63plmb+EXMnLzFtLG+3MUfMbj5dTP6c8wf/kxfWsUPzY2VXc/UhOCZ1ZaWGl1cG1nsw/Ufto8FdMIep6+0japx9LZ9X5bz1cb/1MkdWF+WnHbK2v62Mqypq8ImYsziU9ARfXNy0LBxc+qaf7WTO/+zh/hv+DYu4yopgxlw5+ab2Ny/fuwOV7duDhp5a9rwOStQ8A4MSFtYwe+7/+5EHc+qFjAID3fvZb+MkPfBmfe+gpAMAD310CAFyXcv3XCc6fBOd/fnVzLMVeQIO/og/07e0TS6ln/eupvcPWwV1Vq4p5XieKLcWMl/Ypa/LiXM54ca1dvcM38o9LWjq7rylLEkwSEjFjvpEH/8W5EAd2zVlUmAuT+ccMPJFOAH/x7bO4sNbGRjvCk0sbiBl4x0e+gW89tYzPf/M0Fpohbrh6PwBg13wDV12ygLkwUfuYP+fC2iYu2Tn6Yi+gwV/RBwajfexz1LtejUEquuLiWhu7UyfJKC7aHfvQidg6rhbtEzN2pksSrm9GPXT+1Tn/MM3gpYKnrG8hivNi9XwzuQtZ2uhgcS7E/sU5qwjuwnD+APDomVUce+x8RpGdWW7hzHILNz7zEgQE3Pbnj+DPHj6N73/2Aexo5h3Q//zVz8Y/+N6rEFDydzNzSvvMl153mNDgr6iNukVb6e3TD4WjBd+tx8X1dsY1J7RP7wm9LZqkYuZs4RYJIkp07B7Ofy7NthN312R7T51/pzvn76V9enD+MvNfaXWwONfAvsVmz8z/WYcS/v6xc6v4/DdPZ/tOL2/g9NIGnn/FHrzp6FX4+D3fxYkL6/ih511qneMtLzuCN7z0cKr2Yay0OmhHrJm/YnpQ19snb+ySa7Iq5z9OtKMYO1JfHrnsYczJ4iL3nrhYfE0nd8UsU/sACfUjgzqnKq8sUMey9uPx9hGb2iUFX/OZCL20T4naR3z2zEQEJC6b+xfnLCrMxZNLG3jeFXuwb7GJR8+u4vMPn8Zle5KM/dGza1jdjHDp7h34n7/vmqzi8EPPvdR7LhP8TZe1cv6KqUF9S+etoX2U8986RDGj2TD8vE37/MGxJ/DG275kWSMAwhI5LfwGJdFDdrACojgb5t77mc7f2+QlaR//Z8383uzh6gnkyp5c5w/LkG6xGWLfYhNPr7e9SQkz49TT63jG3h249uBOfOKek/jOmVW8JbVteOBkIhG9dPc8rjm4Ez/2PVfgpUf24Rn7/H78Zg3jc6uJodw4rB2AAYM/Eb2JiB4gopiIjortryGirxPRfenjD4t9N6TbjxPR+4hK0gfFxKJ2wVc016il82QgjtlqkJIT7OpmhHbEWeA1ML+b5RV99g5AEtxkUDeUkgnUhu82z13If3Mvbx9j7MZW8LfPZygeOcHNi8x/cS7EvsU5xJw3fUk8vd7GRjvG5XsXcO2BnVhudXDT8y/D23/o2QgDypQ9l6Z3Ar/xD16Mj976iuIbk4JSzn+c1g7A4H7+9wN4A4D/19l+FsDfY+aTRHQ9gDsBXJnuuw3ArQC+DOBTAG4G8OkBx6EYIfpdzCW26AUt+I4TFg3DYoU15tLir1wMJYr9nD9QzPxzfr442fSSem72cvWk/O7FwJ0odjRDbLRjL+cPAIvzjcxe4eJaG/ucYGyUPlfs3YFrDiyiHTN+5Y0vQjMMcHDXHB48lQT/Q7uT4N/s0a2bqH0Yq63kzsr0GYwaA2X+zPwQMz/s2f4NZj6Z/voAgB1ENE9EVwDYw8x3c/If+hCAWwYZg2L0MN+t2t4+kvOvEfz7mTAU3RExZ0FKFnxtOa4/8zecfdfMXxD35qmkaLrbO8hrlnP+ASFb7Lxbk5fR9Mt6k+T8De0D5LYXn3voKfzor38B65tRpvS5fO8O/MjzL8N/fstLsJD2LFy6e0em+rl09w7v++EiIELEnL1H47B2AEbD+b8RwDeYuYUk+z8h9p1AfkdQABHdSkTHiOjYmTNnhjxMRVXUlXrmtI89EVS+XmyCUZ1RKrohjlnYL8teivz9drNyE4jN2rqlBd+ArEzc/M+bIghXzfwzbx+PeiggyuSm3Tj/+aZD+zBbnP9CSvsAue3F/d9dwreeWsGXHjlrZf4uLs2yfapszhYQIY7zekejZBIdNnrebxDRXQAu9+x6NzN/vMdrXwjgPwH4UbPJc1jpV5qZbwdwOwAcPXpUv/oTgtqLuXhpnzrXs6+rGBwxc77+raR9YvZaHwN5sMpon7LMn8hS8eTe+/mdhjl1ZwBvn4Aoo57k/kLwTwN9J601MOcTAgAszjUy3t0scrPSSiaBz3/zNOKUJjq0q6jHN1TPoV3zqFq+DCi924pN3WJCgz8z39TPiYnoMIA7ALyVmR9JN58AcFgcdhjASfe1islGRvtUPF7eKai3TzlOXlzHZ+5/Ej/9ymuHfq0ozmkfdv4vJmt347LM/GPunvl3YsZffvss2nGMl16VdLk2g/xOo1tTmdzSzduHBO1TtoA7kGTWpvfAHFco+C6ktE8qvzRUzp8++BSWN9q45cVXeukZk/kf2lON8gHymoihvMaV+Q+F9iGifQA+CeBdzPxXZjsznwKwTEQ3piqftwLoevegmDzkFEFNzp/RJ+df705jWvHJe0/h3//Jg17FyVYjZgjOPy+Y2o14bsFX3B3E5Zx/I0wC7fv/7Dh+8/PHReYv7jTSbV7OX2wr9/O3M3+ftNQMrxkGWe+BOY8b/PcsNEGU214sb+TduxvtGP/LK6/x/q0m6JtJoAqMn/+4M/9BpZ6vJ6ITAF4B4JNEdGe66x0Ang3g/yCie9If0/HwNgAfAHAcwCNQpc/UoYoPjIQJ2tZKXn3QPrOu828bNY1H+77ViDnn/G0Ovjwr72Re/int0oX26cSMjU6EjqCRGsKKwfwrey/m4v+sxXFa8O3S5GWuF6aZfxTH2T5L5z/XQBgQ9i40M8O7lVYHl6eB/fuffQDPu3yP9281Qb9O8CdKJqgs8x9TwXcgjREz34GE2nG3/xKAXyp5zTEA1w9yXcV40W+TV1UDMRdRzetNKySnPmxEQudfoH1K/kdth/MPSxJWQ2u02rEl+2yKQN3d2C3fli/+4x5jMn9kf4P7+rkwwGYnRjMkNNIitI/2McqdfQvNbIWzlY0OnnXpTrzzpusyczYf8uBfj/ZJ3sNU7TOpnL9C4aJuAVYG777UPtuE8/ctMD4sxMxCemnLILOsvIfapzTzT5u8Wp0I841QSD2Ft08XtY/v31zO+RddRN3u3zBIVEFRnLuSurQPAOwTFg8rrQ4O7FrEm9Mu3jIc3r+IRkC45uBi1+MkjL2D+X9PbMFXoXBR15ZZBpPMN75GfOtHITSN6JTw28NAzHBon3x7XpdxxifqNb0KvjEzNqMYjSAoqn1iqfP3ePt4PlhenX9AmXWDHKu5CzCTTTMM0EiL0GZinfME//2LTZxZSSwXljc62DXfW7p5aPc8PvdvXoXD+2sG/zinvGaq4KuYbeTBuF7Bt1/aJ9f5z3b0lzr0UVzL6riVHb6ltI+j8+/a5JXQPrJjWHYUWzbfBQ1/8Zy+Dl9b6unL/G3OX8orXc4fMJl/zvkby+teuPrAzlrZu5F6jjvz1+CvqI26NIzMJNXbpxyyg3aYMOc3tAizoOZiLm3y6ghvn5i70z5RzGh1YotGspu88uPdya7s/+zy+gF1d/U02X0jCDIqKnL2AZL2SWydmRO75WHZLpj3x0yg47I30+CvqI0sGFeUXma0D9vLAFa/3vYo+MplBod6HYeGkRy867sTxYzVVPMuXT17WTonwT+y7iQalr1DsahrUL4Au/2cRMHX1+FrJjej8zdupGYs5rULTUP7zGGl1cHSRgdRzNhVMfOvC6LE1bPT5e5pFNDgr6iNuhbLVZQk3dAPVTSNMIulDJv2KapvnA7fbM1l4CNf+Vu8+tf+HMxcVPv0on06sSUjlQVmGciLi713H3cytiTzN1lzZO1Der008zdqHzHphERohAF2NIPsDsbYMzxxfg3A8AzXwsA008Vj4/sBDf6KPlB3cRUp1xtkJa8Zj/1CRz9s2id59BVgJTUXMePJpY2s0Sl/PWecuw9hQGi1o8wnyPw9psYgLZ2Bota/bJJ3s/tyzt8u+DZStU8nzjn/ICA0A8r4fgA4mNo3PHZuFQAqc/51IdU+mvkrpgqx6AatdLyhfUSQqVO83W5Sz2Fn/kUaxl5q06Z9ktcYu4N8O6OsNykMCKubUXYtV3rJovAKFBU/ZXOffFvM5GOoJ3kK8/q5LPNP1D5RZGf+YUAZ3w/kfvzfOZME/2Fl/rLDVzN/xVShLgfPJYGl+vXs684qZEF1mMhpH4/aR2TH8v+1timCf0rflRV8GwFh3QR/0VjV8Lh6mt8lyjl/O7sngpB62pRQcj3J+QeW8igMCM0wsIN/2qj1nTMrAIYb/JmTAn9YthzaCKDBX1EbdbN3k1UaGkCeowq2yxq+Zd2sW41YBEDzu9d8T0wEbuYfdyv4BoTVdLKQ58ubvOy/scj59y74Mvf29pGcfxjYk04QEBohYUHQPsah8ztn08x/SLSPGYty/oqpQ21vHw/t04+r56zr/KWOfpiQ2W+iObeN3WS/gXluVp0yr++1jONamvl3olgE/5z26Zb5Vyn4GqmnSZx9Us+c8w8QBkHK+SfHBJRsX2zmmf+OZog9OxoZ7bO7QpNXP5Ccf6PMI2ME0OCvqI2toH3qcf7246xiVE1e5n0MRPNTnu3bUl6zfdXJ/E3m7UMYEDY7ebeyOZ8s+CZSzWR71cyfHV7f4vytiSF5lPYODcfSOSRCM7Q5fwC4dE++MtewMv9ANJxp5q+YKtTN3u0mr3SbFnwLMFLPoat9DPWRSiUtrx3B/8u7gFXB+RvJZ3nmn4cVWUNwLZ3zpi+74Fv2b/Zz/uVNXhbtQ8nSkpL2ufrATjzr0l3WNeSCLTvn7Ylhq2Dutsat9lFvH0Vt1PX2kRxyf5x//sjMY+uIHDY6o6Z90iYpZtvYTa7klRV8Be1j6KnS4C82R5zLOmVHMXPuulnI/Ev+/qLOXzR5xfa+5Hq51NOsKyzVPh/86ZcVrmEUP3ONwLKA2EqYJrgoyi02xgHN/BW1Yb6DdTn//mmfYjFvFjEqqafMfkMiy4JB3p2VFXxN8C+nffKwIs9h0z65q2inoPPPF2Jxt+fnLS/4ZlLPhlH7BGiE1RZQMRbNu4ek9AHyu5VOHKvOXzFdqEvD+BZw70fqWeea0wi5WMowEWdFzyR4ysJuYT3f9FjJ+Rs+v1znnz+XOn938Rjp9WONj9la4KTMs596ePtYTV5pti0nPh+M3HNYfD+Qj7nVibXgq5gu1C342jJCs63+9eq+btrQzxKXfV3HUB9BvqqUNHbzGfGtbPaX+UtKqREE6fWSz4ExV3NpH2bOrCcAufB7fowp+Pp0/nnB127ykmqfMpmqoX2GpfEH8smsHWnmr5gy1NX5+2iffrx96r5u2tAe0UpemcUB5WofqciS3b4mMEvOf7NHwddVsJjlKYN08RVTB5grzfztpQ2lB5FB4uefT0BVOH9J+5RR7UbrP9Tgn/497UjVPoopQ92mK3mnULdYDNg0yAzHfqvoOkyY/19G+4igKH13TDMXYNM+PQu+bvBPaSJTYzBdvyY4F+0d7KCYZ/52EmBz/vbrAWDOdPiGwtVT/O0+ZJz/EGkfc+3Njmb+iilDXamnrfZJttXj/LdJ5j9iS2ezvKEs8lrKHy4p+HZ60T5O8I/EZBPktE+zUVyCMbmubdcsVUKFY7ImL/EZcYzrfJl/WdA9ZDj/IWb+hnJqpyudjQsa/BW1IbP3KtSPNILrh/bZPgXfEXH+gvYJCJa9Q+Lbg2wcZqJY9XD+VTN/k9mHZBde88y/yPnLhVqapZl/PgHJz6F55uP8e2X+e3Y0sNAMsXvHcLp7gby5bdyZv+r8FbUhv6vM+Ye5/HhZ8K1P+/iW6JtF5NTLcK9j3s4wVcFYlg6xVPtA0D4559+K8mDugxvQjDqIKNW4cyItzdQ+kRv8pYyTLfdR/zHuAu6G8887fMMgsGifsqBLRHjfW16CZzvNX1sJc+1kjWMN/oopQmE5PXT/AOfyzvqUkXvsLPv7ZN4+I8v8jc+MXcS3jN1M5m/RPqZoWr6Sl4RcqzYIEkfLWBR8ffYOVuYfdMv8ze/5681nxKiJmsLYrZfaBwBe84LLSvdtBZTzV0wt6kovZWCRxcTq1/M/nzWMquBrAnoQJJy5belsewyZ58aoDZC0j//8Rc4/rxEEVGzyKuP8TYBuhEVqJ9H5k7B3KH4m8wXcpbFbd7XPKBCIzL9Z9iaOYhyDvJiI3kREDxBRTERHPfuPENEKEf1vYtsNRHQfER0novfRrPbqzzDshTOqcP7FAmL/Ov/Zjf4mSA77bzRB1HDwbi3G97xeh6+/4BsG6eLlpuBbovbJZJxB3qGbjCc/xnQBm2vZxWBOX2fuHFJjN+5N+4wC5tLTnvnfD+ANAL5Ysv/XAXza2XYbgFsBXJf+3DzgGBQjRl27hbyYWL9HwL3GLAd/ybsP9zrJo3HFjNldySvZLxu+rA7fulLPyHD+lC5kktwBNhvlHb6mBwGQa//a9J/09ok8mb+hfULj7RPFlq/RuGCrfaY0+DPzQ8z8sG8fEd0C4DsAHhDbrgCwh5nv5uTb/yEAtwwyBsXoUTcYe/nkvjn/yi+bOrTj+u9NP5DUBxm1j7fgm0tzJS+fZfIlAdQNaJ3Ip/ZBF87fLubmaw2XH+OrC5k7i2YYFHT+4yQcgiz4z+AavkS0E8C/BfCLzq4rAZwQv59ItymmCHVpGKkbN4dHNeLbdqF9clfP4V5HOlvmtI/ZB+Hq6S8+y6YtHwpqH6HzT4KwXfD1Zf6Jb0/yu5lM/JbOye9WQpKeb//iHABg72IzWcOXe+v8RwH5vk20tw8R3UVE93t+XtflZb8I4NeZecU9nefY0m8zEd1KRMeI6NiZM2d6DVUxItQt+Moib75UoRZ8JeI4D8AjdfUMbKmnfO4utG6wWVPqmdUIAmQFZua8yauo87cXajEZPDufA3sB9+Jn8gVX7MGd7/xBvOSqfQiC/I6j29hHAfn2TLTOn5lv6uO8Lwfw94noVwDsAxAT0QaAPwJwWBx3GMDJLte+HcDtAHD06NEZ/dpPH+SXsFKTl8gqB/b2mdHoLwPg0NU+osmLUruFzLIjFndngiaR6NveIb3TMH9rrvMv2jskSzTaap8i559TKJFzVwAAFADPvXx3cg7T5DUJah8x8Yyzw3coOn9m/gHznIh+AcAKM/9m+vsyEd0I4CsA3grgPw9jDIrhoW7mny/gbgeW6tfLn88q6yPfj2EXfKXiJUhdNuUCLj6pp0EzzJdorEr7SJ1/SJTRW911/nnBt1Gq85eunvnrzWFyFGHaXyDHMi7I921qOX8iej0RnQDwCgCfJKI7K7zsbQA+AOA4gEdQVAMpJhx17RakEVxdO2j3GrPK+bdFNXPYf2Pu559LL7MF3EVjl1T7AEngD4jyzL+M9nG2b2ZqnyTwmYJxuZ+/rfPPvH2cv8Ho/I1NtAEjv7Nxx7TZibqOfRSQ8X5qO3yZ+Q4Ad/Q45hec348BuH6Q6yrGi7rBOOOyRWDpn/OfzeAvLQ5G5edvaJ+Y7QnaN1kDuR9/ZtTWo8lrrpEu02ipfXJdfy9vn5z2CbLtBoYaMuf13Y1awT/MFTbJ2Mcv9QSmOPNXbE/YnH/v431qnzqZ/3bw9pGZ/9DVPhYN04X2idkKzM10IfSemX8a0HbOJWvgWq6eRJmBXTdXT6vg61mtyxSFzXl9CQJ5MuxexepRQMpMp1bnr9ieqJ/558HEoF9L51n19vEZkw0LmeIlKPr5S4WPtH0AUr284Px7FXwX5xJiIVf75AupA3ngK/P2KRR8nffI3HmYXgUD8/bJzN88b/Wwox4FQovzn1J7B8X2hLzlrib1TB7lQt21aB+nuWcWId+bUfn558ZursIneS4tnYF0URSiLHvuVfBdyDJ/eyWvjrgTaASEqLCYS8r5G52/dxlHzjLowKV94vzvy8aerZ6Vy07HBWtck6zzVyhcxCwUGBUClQlmdlGzzvW2QcE3krTPsAu+goYJ0saujOqRd2r2xNsIgrRg2506aWSZv037GFfPtnV9Kvfzd2gfq6graJ8wcGkfZOc3CNMJpN3jrmUUsJq8lPZRTBOY87b0KrHYR/vU0/nn2dKwA+O4MFraRwRjImsxF5f2cdU+CeffXS5pqIyFppv5JzWGSKzp2wio4OdvZJxuwdfN/M3lifwJgpybMrVPD1O6UcCalDT4K6YJSeZfbLwpg4/2qcv5mzuNGU38s4AKjMLYzS7ASrfLSD6PbdrHeOT0dvVMHhcLtI9d8DWGa2vtCHc/ci57fRzbBV9fk5cpCpvzugu4J9YPxQy7V4PaKBB6xjUOaPBX1EbMnEnnKnn7pMdI6946QTwWdxqzSvvYmf9wryXtHcwavtJqW3L+Mqg2wgBBUKXDNwkrecFX1BjE5EEp5/+xrz2Bt/yXL+PMciu7rukJAMoXczExNKCiBYg7MZmxmoLvpOj8w2n181dsT8Qxi8y/wvEm+PepZeeadxrTiFE2ecnVrIKUMskCvlD+xIXMn9AIAsuf3wcTWE3BtyMmizBVFwHG6C3IOP+NdtKAZfj84mIu+TWKnL+jBHJie7Z0Yo/u5FFAOX/F1IJZLqLRPVBJSwdZ1Kxr6Rx61nGdJYzD3sFV3wD23Zlr79AI8pW4ktf3knomwd+4elJaYJZ3AjL4ycVsgqDaAu7mvG7mT/Bn/u0oLkwMo4Zy/oqpRcycuyn2iFNyf7/UhqwxzKrOf6RqH0n7iEw8GYechJLsP/PYSTl/g146f5n5m23SHiIgwnwzD0Gy0GwVfD13meaY5Dyuzp/hzksNkfmPM+AC9h2TZv6KqULM+a1rrwxe7m/H7N3e+3oM31J+s4R+KbF+IFezCsimnOQkZDp/jWpnLgy8jVMu8g7fvMkrs2II8smGCHjPLd+Df3XTcwDkzV65t499PrfgW6bzZ8/Y8syfS8c9KtCEZP5DcfVUzDaSYFw/+Hf6pH0kzTSrnP9oaZ/k0WT+MuC7iqw4BhZ2hFhpdZImrwqZ/6Fd8wgIOLx/AUAScGWglpn/K687iLXNTnY9QOj8HdrH1vnntE/B3iHuwvlHE5D5z7Kls2K2YXP+3Y8t6851mjq7n4M5W491VoN/v/WQfiA7YJO1bbtw/swZd98IqtE+Rw4s4tj//ho8vd4GkPxtoQj+JsM3cc8E95zzd5dx9NE+QuoZFP2fyjL/zU48VqUPoJy/Yoph0zDVM/8q28uOrdNUNo2w6iFDNnYzCp4wIMulE8iLs8k4koKvoX2MpbNBN/rkkp1z1kLlOe2T0ztSrQOIzB9O5l/q55+Pw13MxR3ajvRvWG11xqr0AWxrCeX8FVMFW3ff+1gXrjSv9/W2g9RT0C0j6vDN19SVtI94no7JFG4bYWAFq15Zqwly7YizgJuoi3KdP1A0eEuGQ0VjN4fasTn/fB8zFwL8vsUmAOD86ubYaR+f1fRYxjG2KyumFjH7uy69x3qy2IbjxdILvA0KvjIAD3sZR8vbx5V6iueGhjG0TzMkK6j2ok9seaWhaPL/vSwCA67aB0Ln7+P88yBatHco3pWYxdzPr22OveArJ5+munoqpgUsKAP5exnKvNXrBLg6k820wlAvjYCGnvlnTV4p7bMZ+dU+ZkwZ7RMEVsDvFbck7WM+L6GHNmo4i7pkTV4Z7eNT+9iLuZQt9GKwd6Epzt193MOG1eGrtI9iWmBidtUOX/OFlRlOIwwG4PxnM/ibjHu+EYzU0ll23AJ+2edCKtmsqvYxCMRnJOfnxX6X9nGavMw8kRm7OeIBSSW5PSTkZPc7mmE2iU0S7aOWzoqpQexk/r0y+Mg5Hkjog1pNXtJOYsjF0HHB0D5zjXoTYz+QxmdSfQPYtI+xQlhIG7GaYWDRPr3oE1+Wb78+Pc7l/JmtNXz9xm55UTehfey/zxff96e8/7hpH1X7KKYS5gtYlYM331eZ4dQt+G4HnX87y/zDkbh6ZtJLJwK0PQVfY9DWCPLGK6B65g8IZY+X9nHUPgXap+joKjn/wEP7uPYOALA35f3HnfnL66vaRzE1MN+xjIZBNc6/KXraG0FQk/PfDgXfNPg3g6Gv4Ruxrb6RaHsKvkYm2Ww4Ov8eGbRPGWRJRc1KXYHN+Zcu41jG+QdFb59umf+4gz/VmECHCQ3+ilrIMvmKuvvMSkBmO3VpH65eYJ5WGK59LgyG/jfKomcx+OczT077mIKvzfn30svLYzP7ZWtbGedvu3o2vU1etrePuxiOy/kDueJn3AVfO/NXtY9iSlDg/Hu6eiaPTedWt763T7UC87TCrGY11whGoPYRtI8TCO1uX0P75Dr/PIPvfR0fty1NzdwmL5vzz7f7GgplUZccbx+wX4m0d0Iyf+X8FVOJjPOvaLGcH2/TPnXi27bg/OOcHhsF55/RPk7w8al9ds438LzLd+M5l+0uBOxusO4SBD+fb0seyzh/tyZg7ojMo1QQccldgcQkFnzHyfmrt4+iFkxcCivaO5gvtMv/1sluY+aZ1/lHcZwUVGveFfUDuQazGwjlpc1i582Q8Jl3/iAA4NP3n/K+zgdb1mkePUVgT+af2Dsgvb5d78mbxPLX92ryAiTtM+7gnz+f2syfiN5ERA8QUUxER519LyKiu9P99xHRjnT7Denvx4nofeQj5xQTC5N1VfXXz/oChEwkkXrW1fnP9hq+nSiZ4Fzd/TAQOXy5QdPRnBufH0upVSPzp9QyOrlO8XXmm2/UPJHU+Xv9/Nl6NKeigs6/6O0DAPsmRO0zKzr/+wG8AcAX5UYiagD4MIB/xswvBPBqAO10920AbgVwXfpz84BjUIwQeeaffiF7KFPyycJu8mKuXrzdFt4+UaJoCoIRGLvFft29W3w0Dp+2Qqco2eyGnOsvTjbZhBC6mT8snX8x80+ekJjAymSgEvvSLt/xG7vNAOfPzA8x88OeXT8K4F5m/pv0uHPMHBHRFQD2MPPdnHzzPwTglkHGoBgtYifz70n7sCd7rKgUSo5xC8z1xjstiOI466AddsE3oX2S592yUEP7+OSdVQNo7r9TfF2hw7fEz7/I+duvdxdz8dk7AMD+nWnBd8xcg3W3NYNqn+cAYCK6k4j+moh+Nt1+JYAT4rgT6TYviOhWIjpGRMfOnDkzpKEq6qCo9ulxfJrFNqwMMy3wVQhyRTuJ2Yz+7TjN/F3lyhAQxX7aZ04U5cOA8oXaPS6UVTNWV+VjnSuwj4lE5m8VfJ36kkv7+Bdw92T+E0L7WJPpGGeingVfIroLwOWeXe9m5o93Oe8rAXwvgDUAnyOirwNY8hxb+kln5tsB3A4AR48enc1v/ZShqPOv1uRlqX08C3L3ev2s6/yjiNMF0uuZ3vV1LREcw5LMvxkSNtop7ePL/KvSPs7x8nXmbsBkv0ZmGrO7kpedaLgFX+8C7t1on7EXfKdE7cPMN/Vx3hMAvsDMZwGAiD4F4KVI6gCHxXGHAZzs4/yKMSEPxtU6bl2aCMg1/1XieJFmqjXcqUF7hLSPXJSdrECUT9DNIMAGUtqnh16/G0xm6/f2ybcR5TUG49X/6ucewvnVzazD2M38s8YxshMJZvaYO+TOnuPO/Gdd538ngBcR0WJa/H0VgAeZ+RSAZSK6MVX5vBVA2d2DYgLhqnd6r+RljrcphSqvBaSdxGwv4xjFeeY/bHsHaX/gGu4Z+Go0gN+jpxvcZjLb2yc/rhFQwdL5RYf34Rd+4oXZNc2/nmN7LEXO39/k1QgD7NnRmIDgnz+f2g5fIno9EZ0A8AoAnySiOwGAmS8AeC+ArwG4B8BfM/Mn05e9DcAHABwH8AiATw8yBsVoYSiJqgE8t3QuBpYqksa6TWXTikTqGSAMhr+Yi+3tk2+XE7T0YrLVKcVt3ZCpg7qofcx+dzGX/LjkMY7tzF82edl2z37OH0h4/3Gry+3JdHzjGKjJi5nvAHBHyb4PI6F53O3HAFw/yHUV40Ndbx93skheW92kzZWWzirn347yJq+R0D6OCgewA758bi/gUrPg20XtQ04GnHP+fnrIlXpKczqb9vFz/gDw7Et34dCu+UpjHxakp9E4JyLt8FXUQl1vH/OFtVw9w+qBvMD5z2jqH8VJk9coCr5ycZwy2kc+9y04Xlvn72nycmWmZk0Dt0nLPM85f7O9jPbxSz0B4LZ/9NKxF3yB5H0YN/2kwV9RC3ULsC5tI19bhfYx/O6s6/yN1HMUmX8U2w1SBpYctyTzr63zD5zHEtqnEVDmb+Q2aRElBeFSb5/AXcylXNEz3wgrjXvYCGi8Sh9Ajd0UNVHX28dH+1RVCsnzz7rOP/P2GYHOPy5t8iqhfWS2Xjfz7yL1lLEvDChzNvVl7oGQc/qlntUy/0mBXKxmXNDMX1EL/Xr7yE7GZh+0z6x7+7RTbx8iGrq9Q8zs1etLqmcuLAZ8oJ/M376O7e1j14FsYzf7/EbO+fi5tWwBIakgcl09x13U7YWAyJpsxzKGsV5dMXUoePv0RfvUL/jOfubPaKZqn1FYOvton6anEQ+w6Ym84FvtWi7XX5b5S86fUSzYEhHWNiP86G98AX9w7ES2zZzHXcB90jP/SeD8NfgraqEoveweqCLnTkG+tgq3vV28fTpRnAWEoat9pKWzR4WVPPcXZt0Cbi+4Es8yU7Mw5fyZ2VppLB8DsNrqYKMd48xyyxpXUe1TLvWcFJBy/oppQ11vn4wm8mSSVVQtdZvKphWZq+co7B3iCrRPw8/516V93Izft4A7kHwmoogLpm3y2FZqNNfqROm2ZB8VaJ/xWzj0QhjQWO2cAQ3+ipqo7e1jjN08ap869g6zrvM3Hb6jyfylNYK/4OtbfF0+r5/5F2kmeYow5fzdBi6DgAgb7SToZ55DGaWEQsF3wmN/wvmPsbsX0OCvqImCt0+PLNVP+9Q3dtsu3j6JvcMImrw8HbdNz/8oOaYY/OtaOvte5951RHFc0PAbECHL/DfSzF9OYC7nPw0FX+X8FVOFYgG2+/G+xVwyC98qwT/T+W8Pb58woKErmqISzl8WfF17Z4N+F3Mxh5fRPmFAVubvnj6hfZKg33Iyf9fV06wHMMlQnb9i6tB/h2+RW64n9bTPN2vIvX1GkPkzrGKpgWvpbOBr8qpr75BPNvk+t8Gs05Xzz+mejYzzz+9e2KF9poHz18xfMVUo6vy7H28CWWhl/vWbvJLFPWaX8+/EMZph0sk6Cm+fTCMvIkCZ1FMG7PrGbumjZ7IhJ/OPenD+WcE3y/zz17rePpOf+ZNm/orpQlHn3yvzT/b7Mv9qnH/ymCwGPvzu13GhEyVUTDgqtY9Hd2+tudDT0rnatQpqH+tc+XFJk1dsTfYSRISWKfhmnH9+7qnj/IMJWFdgrFdXTB1MYKpqseyqg4Disn3dX59ngoHD7c4SNqMYc2E4Mp2/u7Yu0MXewafzr1nwzXT+ZVLP0GT+KIzLvL4s8098f/Jjp4PzV7WPYspQ19sno31kYKlh1SB9XIhmt+Db6sSYayQ6f+bh0lsy+MvAPmfdnZX5+RfvGLrBnSxKg3+QrBnsmrbJY43Us1Xg/Kut4TtJCFXto5g21Pf2SWkfT4dvHamnyfxnMfYzMzY7MeYbQeGu6C+/fRbnVze39Ho27ZNvb3hst4HBMn/zOfHZSZBVS7Azf1/BN5N6ZmsL56911/Cd9OBPBG3yUkwX6nr7ZLSPRzpYZyUvSjOlWfTz30zXbZxviuCfTgg/9btfxe9/9fEtvR5zns0TFSdlwDbiG8jYzQn6vm5hwMf52+chT+Zvxu4WyaehyUvVPoqpQ12LZV+TVzOsrvbhAu1Td8STD5PRzjfCLFgyA2ubHUQxY6XV2dLrRczeYGw7r26tzr9Xk1fO+eeTvUQQIHP9bEd2UTi5I3TVPpMd/SdB7aOWzopacJfQ68fVs45Vg0v7zCLnbwqYc40AnfQuQAZ9s3+rYHv75NstCw6PHQdQX+fvWjrbUs/8ONPk1c3bp3Buyh9t2mfyC77/+BVX48DOubGOQYO/ohbkl7OK7j7vCC76+dexdJ5lnb+hMeYbAVrtnPZZ27Rpjq1CHHMhKAPuGr7+DL12wdcJ+t2M3TpRd52/i24F30mXev7ky68e9xCU9lHUQ91MPJOGWoqR5GNXj/OfXannZkb7BFlAi2PGapr5b2xx5i/97m17B5uDN/DSPlX9/J3CclmHb27sZvbZwdsXy6W3j1RIxbH/eIUNDf6KWrAz8d7B2Ef7NGvQPrn0jwrL9c0KWiL4y2L4ams4mb/l7SNpnzQyu0oU14YBqK/zryL1jOI4SxZ83j5l584mzPSjMQ1+/pMADf6KWrCCL9Xx9ilmkv3QPrOY+VsFX6H2Wd3sWPu3Com9QzEYm4AfphN7st8uvvpe1w3m305UnDTkKRphd87fdzVXSWQ+i9OwktckQIO/ohZkJp5w8N2P9y3gXkvnH7s6/9mL/sa2YL4RZJx4HEPQPlvM+XOZvUOQbfPZMcjfKxu7OTRR3lnsruHrcP5OZOpa8HWUZwzN/KtAg7+iFrJMPEg5+B6puOvtIzPJSpbOlrfP8Ne3HQdsnX+yLWbGalbw3Xq1T9F2QZq2FYO1Qdn2Mrh3Cj6bh+S8gWPsVoXzt89tPk7T4O0zCRgo+BPRm4joASKKieio2N4kog8S0X1E9BARvUvsuyHdfpyI3kf6X5oq2C6bVTj/5FFmlWH2Za3n7eP6ts8KMqlnmOv8I1Hw3XLaR+jgZSYvFTllmX/Z9jI0nDsMX63BHNfpsphLL6knkCcG0+DtMwkYNPO/H8AbAHzR2f4mAPPM/D0AbgDwM0R0TbrvNgC3Argu/bl5wDEoRoicg0clrx3X/19SCnGFmJbfaRCCYFalnnnmnxcvGWuZzn8YtE/yXGbPWXYuuk/dZq7axm4lOn83uDfCZGKPBM1nn8dz7kLBV3L+Gv17YaDgz8wPMfPDvl0AdhJRA8ACgE0AS0R0BYA9zHw3J9/iDwG4ZZAxKEYLE3yNxXLvNXxttU8Q5Lfw/Xj7zKLaZzMSnL9U+4yU9iHvBO3aONQu+GYZv/16d1IxdwjtyF6ly72ub1v+eTKPmvlXwbA4/z8EsArgFIDHAfwaM58HcCWAE+K4E+k2L4joViI6RkTHzpw5M6ShKurAXVylH9rHzdSqXC/38+9z4BMM2eEri5erQ8z8AyeDDwPKl1oMqLDfIFME1dT5u2ofNzib3o88+Nv7feywHC8gdf6T3+Q1CejZ4UtEdwG43LPr3cz88ZKXvQxABOAZAPYD+Iv0PL7/SOnXmZlvB3A7ABw9enQGv/bTB0PVVM3EXW8fmWH25+0zex8DKfU0GXEUI8v8N4bA+budt0Q2NefuNwhLMvcyFCcZeM+bZ/4l3j6ey7l3ITnnr7RPFfQM/sx8Ux/n/YcAPsPMbQCnieivABwF8BcADovjDgM42cf5FWOCzPyrFGA5dViUC3m7uuxq18s7OWcN0t7BBEer4LvFmX8klnGU2bP0+8mN3+zXZhx+VamnQy9JJ04Jc0dRi/bJJpLkUdI+Gvt7Y1i0z+MAfpgS7ARwI4BvMvMpAMtEdGOq8nkrgLK7B8UEwgRfE8Sr+PmbrB0w9EL1zL/Y5DV70d9r78DDUftkfRNuRk5kUTNmv7vaVL+Zv8v1u5OHyfw3S2ifbpm/GTdbBd9Kw9vWGFTq+XoiOgHgFQA+SUR3prveD2AXEjXQ1wD8LjPfm+57G4APADgO4BEAnx5kDIrRoij17EH7xPaqRRbt05e3z+wF/1YnRkDJmgdybWTT4duJOXP7HBSGhivSPuSnfZwI4bNm7oaCsVuJMZzh/M1EWNT5+wq+9pjsgq9G/14YyNWTme8AcIdn+woSuafvNccAXD/IdRXjQ127BUP7SHVJHdqn6O3T99AnFq1OjPlGCCAPqlHMWGtF1jGNqlXWLnAtuSW9I4NpmdSzrBBcBjNkSfvJ6xq4ah83dvsu564OZiY21iavStAOX0UtSA6+itGaS/sY1U6yr8r1kF5vhi2d2xHmm7kaCkjeN7mIy1ZRP3nBvkjHyAkhC9Il9Ex92if53VX/GIRO8C/j/BeaYWFbRiOagi9U6lkFGvwVtWDp/IPe3j5RbBcTQzER1KF9cnVR/2OfVGxGMebSFFmqfdY2I+ycS4LdVjl7ZrRPwWuHLD6+NPMv4ezLkKm8nPMVMn9T8O347R3M7zvnG2KbvU/aOyjt0xsa/BW1IDt8K/n5e2gfyWtXvZ7x9plJzr8d55l/+o3sxDFWNzu4ZFey2tNWefq73jmSi5d3AWUrdi3MhWiGhN07qjHGZZ29RalnyvmX6vyTR3ndcldPzfyrQFfyUtRCXW8f460uv/x1aJ9t4e0jOH8TdNdaEZiBS3bO44nz61uW+ecuqXZBN6D8uVT7uEF613wDf/IvfgDXHFysdD33DqK84JuqfQy9VeD8Teaf0z5SQQYkdzXMrJx/RWjwV9SC5OCpQiZuFg7Jgkzf9g4zzPl3Ysw3UtonDWRLG20AyNZ53ap1fE0jlBuEg8A/QfsKu8+9fHfl67mF3jIVUW97h+Rx51wx85dSz7L1ABRFKO2jqIWM8w9M01Uv2ieniADbMbKan3/yWFVaOo1odSLMNQztk7w3yxtJsXf/oqF9tijzF0Z5gJ2Zyy7c3N55sCDqTiJU1uFbsclr13w32seuESm6Q4O/ohYK3j49ElJOF9OW3G++YEkfOv+tdTqYCFiZP5ngn2b+Kee/ZWofJzjm0kvKFTky8x8wiLp3GGW2ETnn7w/eJrvvVvCNmQuTm6IcGvwVtVC34BvFnMoIk99l8beWt08w294+mc4/C/5J5n/Jzq0N/hnt4wThOrRPHbgdvS4NlB3ncP5l3j4y+LvFY7kYjKI3NPgraqGut0+B9gkou/VXb58Em51Y0D7JNsP5X7LFtE/ml++RXkqHz7LCbF24On95pyFRpH2c82S0j9T524/MUM6/BjT4K2rB9fbpYsoKwEg9S2ifGlLPIL17mMXMrtWJPAXf4WT+bnAsk3puWeZvFEQF2sc9rgfnn55n13wz3+b8DQnto5x/VWjwV9SCvYB7hcw/TtU+WeZXt8N3G3j7tItSz4z2yTj/PPPvRDF+/o77cPz0cu1ruU1esgCbO3bWX7GrDK4ip+yOolnw87f355x/scM3yO4ki30MinJo8FfUQl2XTUP7UCoNldYB6u2ToNWRTV5p5r9eLvV8/PwaPvKVx/Frd36r9rUiR+fvs3G2ir8DBn9TyHVtHcrsHTY7+WQvYca7ONfIrUIc5VDC+cP7ekURGvwVtVD09ul9vEX5CHqhShK/Hbx9NjtRwd7BcP770+C/ITL/c6ubAIA/ffBJPHF+rda1OMv8i7SPbMRyawL9IreRkNuoQMsYzn+zh85/rhGgGdo+SPnniTMWUjP/3tDgr6gF126hkp+/CCTSOjiqIfWcZW8fmfmb9+bMUgu75xvYlTY1ycz/7HILQPK/+OCXHqt1rcihRbIsmuyu3jLf/brweQHJmkK2zXD+xtLZXUQmPb4ZEuaz4G/vU51/PWjwV9QCC9+USt4+sa30kPSCevsk76fP0nm51cGhPfMIAsJcI7AKvmfTzP+lR/bhU/edqnW9Au0jJmbrLmDLMv8ix09UDM69Of/kcb4RoNlwM/9kn1Xw1ejfExr8FbXg0ji9mq7c4w13n+zrfb1Z9/Yxa9bON+xsFgAu3T2f7ZNSz3MrSeZ/4zMP4MmljVoLvZi5sxvtI+syAxd8PZOIXM0t21agfZzzZJl/kFFkmWw0kGofpPs0+PeCBn9FLUi73CpNVzL4uwGmCn8/694+cv1ewA6Sl+7eke4L7cx/pYX9i00c3r+ImIEz6WRQBXnmD+uRyC6g1l2xqww+aae0kjDIlnHs0eQ11wjynghX6hnbyYKiOzT4K2pBLo5dpekqZljdvTKjrMb558fPotSzJdbvBexgazL/Hc3AknqeW9nEgV3zuGJvMjmcenqj8vUihxYxKiy34Jvr8vv6szKYQG2omuSafej809/nwgDN0A76Nu0D7+sVRWjwV9QCi8y/StOVj/YBqlM4RZ3/AIOfQOTB39b5A8Cle3Laxyr4rrRwcNccLk+D/5M1gn+m9hHXMQVeSQVl1gkDptBHr96P97z+erzkqn3ZNh/tk3P+3b19mo0Ac5kVhr1Pcv4a+ntDg7+iFuK4XsE3im3aR/rHV6FwZEdqFZpp2mBoDtfVE3Bpny3K/J1lHJPn+R2AGcNWFXwbYYCffPnV1vrDPqln2NPVM3mcCwPMhXkCYc4HJJ8VbfKqDg3+ilqwOf8qi7nY/LJsLqpE+8SS8589b58C519K+9iZ/6Fd89i70MSOZoAnn16vfL3c2yfflnD8sKiespW8tgKyy9ug9wLuKe2Tcv5yWOZ5FLNlP6LoDg3+ilqwOf+KOn8xWVhKoVpNXphJqaehczKdv5f2CTO1T6sTYWmjgwM750BEuGLvQq3MPy6hfSQlF8g7tCEE/246/55NXqEJ/vb4AdfbR6N/L2jwV9QCi6atKgE8iu0mL6kuqePqaZqQZi74G9onTHhsGbMO7UppH5H5n081/gd2JRPD5Xt21OL8s+Avm67Sjl75f3XtH7YSCedvb8sy/5IF3CXn3wzt4G+eJk1eyXO3SUxRhL5FilqQtE+VTLyM9gmC3quAJa93agwztpiL4fzdDt+5RoA9C0l3ryz4nl1Ogv/B1PDtir07anL++WRqQA7NY6/qNQzaxx/cw4B6WjrPpTp/+XJp76CZf3Vo8FfUQiyCcV1vH2khEBJlssPur7cnm+2i879093wWoHc084Lv2dVE059l/nt34KmljUqrogH+zF8auQUEa2GXYQR/X8HXbN+Meuv8mw7tYy/gDu/rFUUMFPyJ6FeJ6JtEdC8R3UFE+8S+dxHRcSJ6mIheK7bfQET3pfveR/pfmirELFdQ6h2MzQLugP2lryP1rFsnmCa4Uk8zOZpib7IvwEaa+Z9bSTL/Q2nwv2LvDnRiziaFXjB3Tm7HreXrs4WWzj74OH8goX7MnVAh8zd3RGGA+dBf8I1Zm7zqYNDM/7MArmfmFwH4FoB3AQARvQDAmwG8EMDNAH6LiIwR920AbgVwXfpz84BjUIwQtb19nMmibhafvD55TjPY5OVKPYEk4BqZJ2BLPc+umMw/oX0u37sAoLrWP8pqKPk2+/9Czp3AcGgfX85n0z4uLZQ85mofSVtJ2md44541NHofUg5m/lPx65cB/P30+esAfJSZWwAeJaLjAF5GRI8B2MPMdwMAEX0IwC0APj3IOLrhn37wa/jbc/VsbxXleHJpAztTp8mACI+dXcNr3vuF0uMfP7+GVzzrQHJ8YKt9/uTeUzj22IWu1zuz0rLURedWN7teb9rwdOrbbwV/okzpAyRSz4vrbbzmvV/AudVN7GgG2Vq2Ruv/tg//NRbnQvTCSitZJMalfcydQCPI/ZcCyi2ZtxKNIPCetxkGpX78QTaexOjOx/n/8qceyu6gNPT3xkDB38FPA/jv6fMrkUwGBifSbe30ubvdCyK6FcldAo4cOdLXoI5cstP6YikGw3WX7cL3XnMJAOBNRw+DeyzjeN1lu3DLi5N/8b/44WfjsjSj/ZlXPRNfe+x8peu98Bl7AQC3vORKXFxr97zmtOHS3TtwxZ480/+5v/s8vPyZl2S///iLnoGTT2+AmXEdgL9zeF+277mX78ZPvvwILqxtVr7eqxfm8MxDO7Pf//VrnoPrLtsFAPjZm5+LF1+1HwDw8z/2fLzyuoN9/lXl+Oc/9Cwc2Dlf2P62Vz0L33jiAq7ct4C9C01r30/8nWfgYEp1vfGlh3HdZbuzfVftX8A/uvFIpoQ6evV+HL3mEii6g3rdehPRXQAu9+x6NzN/PD3m3QCOAngDMzMRvR/A3cz84XT/bwP4FIDHAfwHZr4p3f4DAH6Wmf9er4EePXqUjx07Vv0vUygUCgWI6OvMfNTd3jPzN4G6y4l/CsCPA/gRzmeSEwCuEocdBnAy3X7Ys12hUCgUI8Sgap+bAfxbAD/BzJJY/wSANxPRPBFdi6Sw+1VmPgVgmYhuTFU+bwXw8UHGoFAoFIr6GJTz/00A8wA+m1bcv8zM/4yZHyCijwF4EEAHwNuZ2ThTvQ3A7wFYQFLoHVqxV6FQKBR+DKr2eXaXfe8B8B7P9mMArh/kugqFQqEYDCqDUSgUim0IDf4KhUKxDaHBX6FQKLYhNPgrFArFNkTPJq9JARGdAfC3fb78IICzWzicWYW+T9Wg71M16PvUG6N4j65m5kPuxqkJ/oOAiI75OtwUNvR9qgZ9n6pB36feGOd7pLSPQqFQbENo8FcoFIptiO0S/G8f9wCmBPo+VYO+T9Wg71NvjO092hacv0KhUChsbJfMX6FQKBQCGvwVCoViG2Kmgz8R3ZwuIH+ciH5u3OOZJBDRY0R0HxHdQ0TH0m2XENFniejb6eP+cY9z1CCi3yGi00R0v9hW+r4Q0bvSz9fDRPTa8Yx69Ch5n36BiL6bfqbuIaIfE/u26/t0FRH9GRE9REQPENG/TLeP/TM1s8E/XTD+/QD+LoAXAHhLurC8IscPMfOLhc745wB8jpmvA/C59Pftht8DcLOzzfu+pJ+nNwN4Yfqa30o/d9sBv4fi+wQAv55+pl7MzJ8Ctv371AHwb5j5+QBuBPD29P0Y+2dqZoM/gJcBOM7M32HmTQAfRbKwvKIcrwPwwfT5BwHcMr6hjAfM/EUA7uLCZe/L6wB8lJlbzPwogONIPnczj5L3qQzb+X06xcx/nT5fBvAQknXLx/6ZmuXgfyWAJ8TvXReL34ZgAH9KRF8nolvTbZelq60hfbx0bKObLJS9L/oZK+IdRHRvSgsZKkPfJwBEdA2AlwD4CibgMzXLwZ8821TXmuP7mfmlSGixtxPRD457QFMI/YzZuA3AswC8GMApAP93un3bv09EtAvAHwF4JzMvdTvUs20o79UsB/+yReQVAJj5ZPp4GsAdSG4tnyKiKwAgfTw9vhFOFMreF/2MCTDzU8wcMXMM4L8gpyu29ftERE0kgf+/MfMfp5vH/pma5eD/NQDXEdG1RDSHpIjyiTGPaSJARDuJaLd5DuBHAdyP5P35qfSwnwLw8fGMcOJQ9r58AsCbiWieiK4FcB2Ar45hfBMBE8xSvB7JZwrYxu8TJYub/zaAh5j5vWLX2D9Tgy7gPrFg5g4RvQPAnQBCAL/DzA+MeViTgssA3JF8LtEA8BFm/gwRfQ3Ax4jonwB4HMCbxjjGsYCIfh/AqwEcJKITAP4dgP8Iz/vCzA8Q0ccAPIhE1fF2Zo7GMvARo+R9ejURvRgJTfEYgJ8Btvf7BOD7AfxjAPcR0T3ptp/HBHym1N5BoVAotiFmmfZRKBQKRQk0+CsUCsU2hAZ/hUKh2IbQ4K9QKBTbEBr8FQqFYhtCg79CoVBsQ2jwVygUim2I/x+JdED+cZ+mIwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -110:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
